import pymysql
import datetime
import time
import requests
'爬虫全部函数，area和Top10文件夹弃用'
# 注意，主播获取前20，分区获取前10存进数据库

# 爬虫地址字典
urlDict = {
    "amusement": "https://www.huya.com/cache.php?m=LiveList&do=getLiveListByPage&gameId=100022&tagAll=0&page=",
    "mobileGame": "https://www.huya.com/cache.php?m=LiveList&do=getLiveListByPage&gameId=100004&tagAll=0&page=",
    "onlineGame": "https://www.huya.com/cache.php?m=LiveList&do=getLiveListByPage&gameId=100023&tagAll=0&page=",
    "singleGame": "https://www.huya.com/cache.php?m=LiveList&do=getLiveListByPage&gameId=100002&tagAll=0&page="
}


# 接口爬虫
def getInterData(url):
    # 页码
    page = 1
    # 爬取的数据
    result = []
    while True:
        # 获取接口返回数据
        res = requests.get(url + str(page))
        # 转成json格式
        response = res.json()
        # 获取详细内容
        artists = response['data']['datas']

        if len(artists)==0:
            break
        
        result.extend(artists)
        print("第%s页" % page)
        page += 1
        # 防止ip被禁用，限制5秒爬一次
        # time.sleep(5)
    return result


# 获取主播Top10函数
def getAnchorData(artists, areaName):
    result = []
    for i in artists:
        # 房间号
        roomNumber = i['profileRoom']
        # 游戏名
        gameFullName = i['gameFullName']
        # 主播名
        nick = i['nick']
        # 热度
        totalHot = int(i['totalCount'])
        # 时间
        date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        # print(roomNumber,areaName,gameFullName,nick,totalCount,date)
        dict_data = {
            "roomNumber": roomNumber,
            "areaName": areaName,
            "gameFullName": gameFullName,
            "nick": nick,
            "totalHot": totalHot,
            "date": date
        }
        result.append(dict_data)
    return result


# 计算排行榜前10的函数(list格式)
def computTop(dict_data):
    # 基于热度排序，顺序为从大到小
    dict_data.sort(key=lambda x: x["totalHot"], reverse=True)
    # 返回所有主播
    return dict_data


# 计算主播热度热度（dict格式）
def computeTopDict(dict_data):
    # dict_data = computTop(dict_data)
    result = {}
    for data in dict_data:
        result[data["nick"]] = data["totalHot"]
    return result


# 获取分区函数（分区词云也由此获取）
def getAreaData(artists):
    # 返回值
    result = []
    for i in artists:
        # 暂时只填写游戏名+热度，减少接口压力，其余字段后续补充
        # 游戏名
        gameFullName = i['gameFullName']
        # 热度
        totalHot = int(i['totalCount'])
        # 组装成字典
        dict_data = {
            "gameFullName": gameFullName,
            "totalHot": totalHot,
        }
        # 放入数组内
        result.append(dict_data)
    return result


# 计算分区热度和（list格式）
def computeAreaTotalHot(dict_data):
    result = []
    # 遍历传过来的数据，如果在result里面有这个游戏，则热度相加。
    # 没有这个游戏，则将游戏加入到result里面
    for data in dict_data:
        # True代表新游戏，FALSE代表旧游戏
        flag = True
        for res in result:
            # 如果result里面也有data里面这个分区
            if res["gameFullName"] == data["gameFullName"]:
                res["totalHot"] = res["totalHot"] + data["totalHot"]
                flag = False
                break
        if flag:
            result.append(data)
    return result


# 计算热度和（dict格式）
def computeAreaTotalHotDict(dict_data):
    dict_data = computeAreaTotalHot(dict_data)
    result = {}
    for data in dict_data:
        result[data["gameFullName"]] = data["totalHot"]
    return result


# 创建MySQL连接函数（当做固定）
def getMysqlConn():
    conn = pymysql.connect(host='106.13.194.7',
                           port=3306,
                           user='rdsroot',
                           password='abc123456',
                           db='huya',
                           charset='utf8')
    # 创建操作的游标
    cursor = conn.cursor()
    # 设置字符编码及自动提交
    cursor.execute('set names utf8')  # 固定格式
    cursor.execute('set autocommit=1')
    # 设置自动提交
    return conn, cursor


# 保存分区数据
# 变量，数据字典，分区名
def saveAreaDate(dict_data, areaName):
    conn, cursor = getMysqlConn()
    # sql语句（注意要改回来，删除copy1）
    sql = "insert into t_areaHot (areaName,gameFullName,totalHot,date) values(%s,%s,%s,%s)"
    for data in dict_data:
        # 封装成元组进行插入
        add_data = (areaName, data['gameFullName'], data['totalHot'],
                    datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        # 这里会把sql语句中的%s替换掉
        cursor.execute(sql, add_data)
    print("mysql插入数据完成")
    # 关闭连接
    cursor.close()
    conn.close()


# 保存主播前10的数据
def saveTop10Data(dict_data):
    conn, cursor = getMysqlConn()
    # sql语句（注意要改回来，删除copy1）
    sql = "insert into t_anchorHot (roomNumber,areaName,gameFullName,nick,totalHot,date) values(%s,%s,%s,%s,%s,%s)"
    for data in dict_data:
        # 封装成元组进行插入
        add_data = (data['roomNumber'], data['areaName'], data['gameFullName'],
                    data['nick'], data['totalHot'], data['date'])
        # 这里会把sql语句中的%s替换掉
        cursor.execute(sql, add_data)
    print("mysql插入数据完成")
    # 关闭连接
    cursor.close()
    conn.close()


# 单元测试用
if __name__ == '__main__':
    # 娱乐区
    res = getInterData(urlDict["amusement"])
    # 分区操作
    area = getAreaData(res)
    compute = computeAreaTotalHot(area)
    print(compute)
    # # 分区获取前20
    # compute = sorted(compute, key=lambda x: x['totalHot'], reverse=True)[0:20]
    # saveAreaDate(areaName="娱乐天地", dict_data=compute, tbName="t_areaHot_copy1")
    # # 主播操作
    # anchor = getAnchorData(res, "娱乐天地")
    # top = computTop(anchor)
    # saveTop10Data(top)
